# coding: utf-8
import urllib
import urllib2
import cookielib
from HTMLParser import HTMLParser
import re
import xlrd
import xlwt
import string
from urllib2 import Request, urlopen, URLError, HTTPError
import requests
import re
import urllib2
import urllib
import datetime
import threading
import logging
from logging.handlers import TimedRotatingFileHandler


global COUNT
global TIMEOUT
global pro_dict
global pro_data_dict
global pro_certify_dict


def get_page(url, refer_url, description):
    for i in range(1, COUNT):
        try:
            request = urllib2.Request(url)
            if refer_url:
                request.add_header("Referer", refer_url)
            request.add_header("Host", "www.yfci.gov.cn")
            request.add_header("User-Agent", "Mozilla/5.0 (compatible; "
                                             "MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)")
            response = urllib2.urlopen(request, timeout=TIMEOUT)
            the_page = response.read()
            the_page = the_page.decode('gbk')
            return the_page
        except Exception, x:
            if i == COUNT - 1:
                print (description + " " + x)
            else:
                continue

def get_data():
    global pro_dict
    global pro_data_dict
    global pro_certify_dict
    #商品房信息发布
    url = "http://www.yfci.gov.cn/HousePresell/user_kfs.aspx"
    the_page = get_page(url, None, '商品房信息发布')
    str_patt = "href=\'([^\']+)\'>云浮"
    str_patt = str_patt.decode('utf-8')
    pattern = re.compile(str_patt)
    results = pattern.findall(the_page)
    #进入新兴县数据地址
    if len(results) > 0:
        pro_url = url
        result = results[0] + '&page=1'
        #打开新兴的数据列表界面
        url = url.replace('user_kfs.aspx', result)
        print u"获取数据列表："
        the_page = get_page(url, None, '数据列表界面')
        #新兴数据列表的分页地址
        str_patt = "href=\"([^\"]+)\">[[\d]"
        str_patt = str_patt.decode('utf-8')
        pattern = re.compile(str_patt)
        results = pattern.findall(the_page)
        results.insert(0, result)
        url_list = []

        for result in results:
            print u"网址："+result
            url_list.append(pro_url.replace('user_kfs.aspx', result))

        #pro_dice保存的是项目的url地址及项目的referer，名称，及实际地址
        n_loops = range(len(url_list))
        threads = []
        print u"获取项目"
        for u in url_list:
            get_project_dict(u)

        print u"获取项目相关数据"
        #项目详情
        if len(pro_dict) > 0:
            #每个项目的数据,主要是获取里面的分页url
            n_loops = range(len(pro_dict))
            threads = []
            for k, v in pro_dict.items():
                get_project_data(k,v)




#商品房信息发布每一页的列表数据
def get_project_dict(pro_url):
    global pro_dict
    str_patt = "href=\'([^\']+)\'\s+style=\'color:#FF0000\'>([^<]+)[^{290}]+290px\">([^<]+)"
    str_patt = str_patt.decode('utf-8')
    pattern = re.compile(str_patt)
    #每一页对应的项目
    the_page = get_page(pro_url, None, '商品房信息发布列表数据')
    results = pattern.findall(the_page)
    refer_url = pro_url.replace('user_kfs.aspx', 'user_kfs_old.aspx')
    #每一页的项目数据,包括地址，项目名称，项目地址
    for r in results:
        url = r[0]
        print (u"获取项目: " + r[1])
        pro_dict[url] = {"referer": refer_url, "url": url, "name": r[1], "address": r[2]}

#项目列表数据
def get_project_data(pro_url, v):
    global pro_data_dict
    str_patt = "href=\"(User_Presell_beian[^\"]+)"
    str_patt = str_patt.decode('utf-8')
    #开发商名字
    kfs_patt = ur'kfsmctd[^/]+>([0-9A-Za-z\u4e00-\u9fa5]+)<'
    #总套数
    zts_patt = "=\"ztstd[^>]+\">([0-9.]+)"
    #总面积
    zmj_patt = "=\"zmjtd[^>]+\">([0-9.]+)"
    #住宅套数
    zzts_patt = "=\"zztstd[^>]+\">([0-9.]+)"
    #住宅面积
    zzmj_patt = "=\"zzmjtd[^>]+\">([0-9.]+)"
    #非住宅套数
    fzzts_patt = "=\"fzztstd[^>]+\">([0-9.]+)"
    #非住宅面积
    fzzmj_patt = "=\"fzzmjtd[^>]+\">([0-9.]+)"
    #已售住宅套数
    yszzts_patt = "=\"yszztstd[^>]+\">([0-9.]+)"
    #已售住宅面积
    yszzmj_patt = "=\"yszzmjtd[^>]+\">([0-9.]+)"
    #剩余住宅套数
    kszzts_patt = "=\"kszztstd[^>]+\">([0-9.]+)"
    #剩余住宅面积
    kszzmj_patt = "=\"kszzmjtd[^>]+\">([0-9.]+)"
    #已售非住宅套数
    ysfzzts_patt = "=\"ysfzztstd[^>]+\">([0-9.]+)"
    #住宅均价
    xmjj_patt = "=\"xmjjtd[^>]+\">([0-9.]+)"
    #住宅均价
    fzzjj_patt = "=\"fzzjjtd[^>]+\">([0-9.]+)"
    #项目详细信息页面, pro_dict的key就是url地址，值是一个dict
    pro_url_list = []
    url = "http://www.yfci.gov.cn/HousePresell/" + v["url"]
    the_page = get_page(url, v["referer"], '项目详细信息页面')
    pattern = re.compile(str_patt)
    pro_result = pattern.findall(the_page)
    #点击项目进去的分页
    #pro_data_dict保存的是<项目详细信息>页面的楼盘列表信息的地址
    if len(pro_result) > 0:
        r = v["url"] + '&page=1'
        pro_result.insert(0, r)
    else:
        pro_result.insert(0, v["url"])
    for r in pro_result:
        pro_url_list.append(r)
    pro_data_dict[pro_url] = pro_url_list
    #匹配开发商
    pattern = re.compile(kfs_patt)
    kfs_results = pattern.findall(the_page)
    v['company'] = ''
    if len(kfs_results) > 0:
        v['company'] = kfs_results[0]
    #总套数
    pattern = re.compile(zts_patt)
    zts_results = pattern.findall(the_page)
    v['zts'] = '0'
    if len(zts_results) > 0:
        v['zts'] = zts_results[0]
    #总面积
    pattern = re.compile(zmj_patt)
    zmj_results = pattern.findall(the_page)
    v['zmj'] = '0'
    if len(zmj_results) > 0:
        v['zmj'] = zmj_results[0]
    #住宅套数
    pattern = re.compile(zzts_patt)
    zzts_results = pattern.findall(the_page)
    v['zzts'] = '0'
    if len(zzts_results) > 0:
        v['zzts'] = zzts_results[0]
    #住宅面积
    pattern = re.compile(zzmj_patt)
    zzmj_results = pattern.findall(the_page)
    v['zzmj'] = '0'
    if len(zzmj_results) > 0:
        v['zzmj'] = zzmj_results[0]
    #非住宅套数
    pattern = re.compile(fzzts_patt)
    fzzt_sresults = pattern.findall(the_page)
    v['fzzts'] = '0'
    if len(fzzt_sresults) > 0:
        v['fzzts'] = fzzt_sresults[0]
    #非住宅面积
    pattern = re.compile(fzzmj_patt)
    fzzmj_results = pattern.findall(the_page)
    v['fzzmj'] = '0'
    if len(fzzmj_results) > 0:
        v['fzzmj'] = fzzmj_results[0]
    #已售住宅套数
    pattern = re.compile(yszzts_patt)
    ys_zz_ts_results = pattern.findall(the_page)
    v['yszzts'] = '0'
    if len(ys_zz_ts_results) > 0:
        v['yszzts'] = ys_zz_ts_results[0]
    #已售住宅面积
    pattern = re.compile(yszzmj_patt)
    yszzmjresults = pattern.findall(the_page)
    v['yszzmj'] = '0'
    if len(yszzmjresults) > 0:
        v['yszzmj'] = yszzmjresults[0]
    #剩余住宅套数
    pattern = re.compile(kszzts_patt)
    kszztsresults = pattern.findall(the_page)
    v['kszzts'] = '0'
    if len(kszztsresults) > 0:
        v['kszzts'] = kszztsresults[0]
    #剩余住宅面积
    pattern = re.compile(kszzmj_patt)
    kszzmjresults = pattern.findall(the_page)
    v['kszzmj'] = '0'
    if len(kszzmjresults) > 0:
        v['kszzmj'] = kszzmjresults[0]
    #已售非住宅套数
    pattern = re.compile(ysfzzts_patt)
    ysfzztsresults = pattern.findall(the_page)
    v['ysfzzts'] = '0'
    if len(ysfzztsresults) > 0:
        v['ysfzzts'] = ysfzztsresults[0]
    #住宅均价
    pattern = re.compile(xmjj_patt)
    xmjjresults = pattern.findall(the_page)
    v['xmjj'] = '0'
    if len(xmjjresults) > 0:
        v['xmjj'] = xmjjresults[0]
    #非住宅均价
    pattern = re.compile(fzzjj_patt)
    fzzjjresults = pattern.findall(the_page)
    v['fzzjj'] = '0'
    if len(fzzjjresults) > 0:
        v['fzzjj'] = fzzjjresults[0]
    print u"项目公司:"+v['company']+u"  项目名称:"+v['name']+u"  项目地址:"+v['address']



COUNT = 2
TIMEOUT = 60
pro_dict = {}
pro_data_dict = {}
pro_certify_dict = {}

logger = logging.getLogger("root")
formatter = logging.Formatter('%(asctime)s level-%(levelname)-8s %(message)s')
fileTimeHandler = TimedRotatingFileHandler("fdc.log", "H", 1, 10)
fileTimeHandler.suffix = "%Y%m%d-%H%M.log"
fileTimeHandler.setFormatter(formatter)
logging.basicConfig(level=logging.INFO)
fileTimeHandler.setFormatter(formatter)
logger.addHandler(fileTimeHandler)
logger.info("开始于：" + datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'))


print u"读取数据开始"
get_data()
print u"读取数据完毕"
print u"长度为："+str(len(pro_dict))
#生成文件
f= xlwt.Workbook()#创建工作簿
sheetname = "信息"
sheet = f.add_sheet(unicode(sheetname,'utf-8'),cell_overwrite_ok=True)
sheet.write(0,0,u'开发商')
sheet.write(0,1,u'项目名称')
sheet.write(0,2,u"地址")

kl=1
for k, v in pro_dict.items():
    sheet.write(kl,0,v['company'])
    sheet.write(kl,1,v['name'])
    sheet.write(kl,2,v['address'])
    kl=kl+1
f.save( u'云浮市商品房信息.xls')
